#!/usr/bin/env python
# coding=utf-8
'''
@Author: John
@Email: johnjim0816@gmail.com
@Date: 2020-06-09 20:25:52
@LastEditor: John
LastEditTime: 2022-06-09 19:04:44
@Discription: 
@Environment: python 3.7.7
'''
import random
import numpy as np
import torch
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F
from PER import Memory_Buffer_PER

class ReplayBuffer:
    def __init__(self, capacity):
        self.capacity = capacity # 经验回放的容量
        self.buffer = [] # 缓冲区
        self.position = 0   # pointer 
    
    def push(self, state, action, reward, next_state, done):
        ''' 缓冲区是一个队列，容量超出时去掉开始存入的转移(transition)
        '''
        if len(self.buffer) < self.capacity:  # not overflow then add None to wait for a new data
            self.buffer.append(None)
        self.buffer[self.position] = (state, action, reward, next_state, done)
        self.position = (self.position + 1) % self.capacity  # if overflow start with the first pos
    
    def sample(self, batch_size):
        batch = random.sample(self.buffer, batch_size) # 随机采出小批量转移
        state, action, reward, next_state, done =  zip(*batch) # 解压st at rt st+1 done
        return state, action, reward, next_state, done
    
    def __len__(self):
        ''' 返回当前存储的量
        '''
        return len(self.buffer)

class Actor(nn.Module):
    '''
        输入状态输出动作
    '''
    def __init__(self, n_states, n_actions, hidden_dim, init_w=3e-3):
        super(Actor, self).__init__()  
        self.linear1 = nn.Linear(n_states, hidden_dim)
        self.linear2 = nn.Linear(hidden_dim, hidden_dim)
        self.linear3 = nn.Linear(hidden_dim, n_actions)
        
        self.linear3.weight.data.uniform_(-init_w, init_w)
        self.linear3.bias.data.uniform_(-init_w, init_w)  # 均匀分布初始化参数
        
    def forward(self, x):
        x = F.relu(self.linear1(x))
        x = F.relu(self.linear2(x))
        x = torch.tanh(self.linear3(x))
        return x

class Actor2(nn.Module):
    '''
        输入状态输出动作
    '''
    def __init__(self, n_states, n_actions, hidden_dim, init_w=3e-3):
        super(Actor2, self).__init__()
        self.conv1 = nn.Conv2d(3,16,kernel_size=3)
        self.pooling1 = nn.MaxPool2d(3)
        self.conv2 = nn.Conv2d(16,64,kernel_size=3)
        self.pooling2 = nn.MaxPool2d(3)
        self.conv3 = nn.Conv2d(64,128,kernel_size=3)
        self.pooling3 = nn.MaxPool2d(3)

        self.linear1 = nn.Linear(128, hidden_dim)
        self.linear2 = nn.Linear(hidden_dim, 128)
        self.linear3 = nn.Linear(128, n_actions)
        
        self.linear3.weight.data.uniform_(-init_w, init_w)
        self.linear3.bias.data.uniform_(-init_w, init_w)  # 均匀分布初始化参数
        
    def forward(self, x):
        x = self.pooling1(F.relu(self.conv1(x)))
        x = self.pooling2(F.relu(self.conv2(x)))
        x = self.pooling3(F.relu(self.conv3(x)))

        x = x.view(x.size(0), -1)

        x = F.relu(self.linear1(x))
        x = F.relu(self.linear2(x))
        x = torch.tanh(self.linear3(x))
        return x

class Critic(nn.Module):
    '''
        输入状态、动作输出评价
    '''
    def __init__(self, n_states, n_actions, hidden_dim, init_w=3e-3):
        super(Critic, self).__init__()
        
        self.linear1 = nn.Linear(n_states + n_actions, hidden_dim)
        self.linear2 = nn.Linear(hidden_dim, hidden_dim)
        self.linear3 = nn.Linear(hidden_dim, 1)
        # 随机初始化为较小的值
        self.linear3.weight.data.uniform_(-init_w, init_w)
        self.linear3.bias.data.uniform_(-init_w, init_w)
        
    def forward(self, state, action):
        # 按维数1拼接
        x = torch.cat([state, action], 1)
        x = F.relu(self.linear1(x))
        x = F.relu(self.linear2(x))
        x = self.linear3(x)
        return x

class Critic2(nn.Module):
    '''
        输入状态、动作输出评价
    '''
    def __init__(self, n_states, n_actions, hidden_dim, init_w=3e-3):
        super(Critic2, self).__init__()
        self.conv1 = nn.Conv2d(3,16,kernel_size=3)
        self.pooling1 = nn.MaxPool2d(3)
        self.conv2 = nn.Conv2d(16,64,kernel_size=3)
        self.pooling2 = nn.MaxPool2d(3)
        self.conv3 = nn.Conv2d(64,128,kernel_size=3)
        self.pooling3 = nn.MaxPool2d(3)

        self.linear1 = nn.Linear(128, hidden_dim)
        self.linear2 = nn.Linear(hidden_dim, 128)
        self.linear3 = nn.Linear(129, 1)
        # 随机初始化为较小的值
        self.linear3.weight.data.uniform_(-init_w, init_w)
        self.linear3.bias.data.uniform_(-init_w, init_w)
        
    def forward(self, state, action):
        # 按维数1拼接
        # x = torch.cat([state, action], 1)
        x = self.pooling1(F.relu(self.conv1(state)))
        x = self.pooling2(F.relu(self.conv2(x)))
        x = self.pooling3(F.relu(self.conv3(x)))

        x = x.view(x.size(0), -1)

        x = F.relu(self.linear1(x))
        x = F.relu(self.linear2(x))
        x = torch.cat([x, action], 1)
        x = self.linear3(x)
        return x

class DDPG:
    def __init__(self, cfg, n_states, n_actions):
        self.device = torch.device(cfg['device'])
        self.critic = Critic2(n_states, n_actions, cfg['hidden_dim']).to(self.device)
        self.actor = Actor2(n_states, n_actions, cfg['hidden_dim']).to(self.device)
        self.target_critic = Critic2(n_states, n_actions, cfg['hidden_dim']).to(self.device)
        self.target_actor = Actor2(n_states, n_actions, cfg['hidden_dim']).to(self.device)

        # 复制参数到目标网络  保证两者在初始化的时候参数相同
        for target_param, param in zip(self.target_critic.parameters(), self.critic.parameters()):
            target_param.data.copy_(param.data)
        for target_param, param in zip(self.target_actor.parameters(), self.actor.parameters()):
            target_param.data.copy_(param.data)

        self.critic_optimizer = optim.Adam(
            self.critic.parameters(),  lr=cfg['critic_lr'])
        self.actor_optimizer = optim.Adam(
            self.actor.parameters(), lr=cfg['actor_lr'])
        
        self.memory = Memory_Buffer_PER(cfg['memory_capacity'])
        self.batch_size = cfg['batch_size']
        self.soft_tau = cfg['soft_tau'] # 软更新参数
        self.gamma = cfg['gamma']

    def choose_action(self, state):
        # 将状态压缩成向量 传入actor生成动作
        state = torch.FloatTensor(state).unsqueeze(0).to(self.device)
        # state = torch.FloatTensor(state).to(self.device)
        action = self.actor(state)  # 只传入一维的向量 action-->[1,1]
        '''
        返回一个new Tensor,只不过不再有梯度。

        如果想把CUDA tensor格式的数据改成numpy时,需要先将其转换成cpu float-tensor随后再转到numpy格式。 
        numpy不能读取CUDA tensor 需要将它转化为 CPU tensor
        '''
        return action.detach().cpu().numpy()[0, 0]  # 通过索引将数组转化成scalar

    def update(self):
        if self.memory.size() < self.batch_size: # 当 memory 中不满足一个批量时，不更新策略
            return
        # 从经验回放中(replay memory)中随机采样一个批量的转移(transition)
        idxs, state, action, reward, next_state, done = self.memory.sample(self.batch_size)
        # 转变为张量
        state = torch.FloatTensor(np.array(state)).to(self.device)
        next_state = torch.FloatTensor(np.array(next_state)).to(self.device)
        action = torch.FloatTensor(np.array(action)).to(self.device)
        reward = torch.FloatTensor(reward).unsqueeze(1).to(self.device)  # 行向量
        done = torch.FloatTensor(np.float32(done)).unsqueeze(1).to(self.device)
        '''
            critic 更新 actor
            target_actor 和 target_critic 更新 critic 
        '''
        policy_loss = self.critic(state, self.actor(state))
        policy_loss = -policy_loss.mean()

        next_action = self.target_actor(next_state)
        target_value = self.target_critic(next_state, next_action.detach())

        expected_value = reward + (1.0 - done) * self.gamma * target_value
        expected_value = torch.clamp(expected_value, -np.inf, np.inf)  # 每个元素的范围限制到区间 [min,max]

        value = self.critic(state, action)  # (state, action) --> next_state
        value_loss = nn.MSELoss()(value, expected_value.detach())  # 用于更新critic
        
        # 计算TD-error 用于更新 权重
        with torch.no_grad():
            TD_error = value.detach().cpu().numpy() - expected_value.detach().cpu().numpy()
            self.memory.update(idxs, TD_error)

        self.actor_optimizer.zero_grad()
        policy_loss.to(self.device)
        policy_loss.backward()
        self.actor_optimizer.step()

        self.critic_optimizer.zero_grad()
        value_loss.to(self.device)
        value_loss.backward()
        self.critic_optimizer.step()

        # 软更新
        for target_param, param in zip(self.target_critic.parameters(), self.critic.parameters()):
            target_param.data.copy_(
                target_param.data * (1.0 - self.soft_tau) +
                param.data * self.soft_tau
            )
        for target_param, param in zip(self.target_actor.parameters(), self.actor.parameters()):
            target_param.data.copy_(
                target_param.data * (1.0 - self.soft_tau) +
                param.data * self.soft_tau
            )
    
    def save(self,path):
        torch.save(self.actor.state_dict(), path+'checkpoint.pt')

    def load(self,path):
        self.actor.load_state_dict(torch.load(path+'checkpoint.pt')) 